{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0428.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/0428.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZanCMC7kC0S8",
        "colab_type": "text"
      },
      "source": [
        "## 4.1 构造模型"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QN3w3pabDNxI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install mxnet d2lzh "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gYHX92QHEENa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "class MLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(MLP, self).__init__(**kwargs)\n",
        "        self.hidden = nn.Dense(256, activation='relu')\n",
        "        self.output = nn.Dense(10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.output(self.hidden(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EhAbbd0DHsWI",
        "colab_type": "code",
        "outputId": "fb175429-4f13-425e-dfbe-5ae810c8700a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "X = nd.random.uniform(shape=(2, 20))\n",
        "net = MLP()\n",
        "net.initialize()\n",
        "net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.09543004  0.04614332 -0.00286654 -0.07790349 -0.05130243  0.02942037\n",
              "   0.08696642 -0.0190793  -0.04122177  0.05088576]\n",
              " [ 0.0769287   0.03099705  0.00856576 -0.04467199 -0.06926839  0.09132434\n",
              "   0.06786595 -0.06187842 -0.03436673  0.04234694]]\n",
              "<NDArray 2x10 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1mxHmhWH1Tn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MySequential(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(MySequential, self).__init__(**kwargs)\n",
        "\n",
        "    def add(self, block):\n",
        "        self._children[block.name] = block\n",
        "\n",
        "    def forward(self, x):\n",
        "        for block in self._children.values():\n",
        "            x = block(x)\n",
        "        return x "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_NdluCL8IcjI",
        "colab_type": "code",
        "outputId": "6bb43f9f-8bac-4df2-bc72-aec896e3834a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "net = MySequential()\n",
        "net.add(nn.Dense(256, activation='relu'))\n",
        "net.add(nn.Dense(10))\n",
        "net.initialize()\n",
        "net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.00362228  0.00633332  0.03201144 -0.01369375  0.10336449 -0.03508018\n",
              "  -0.00032164 -0.01676023  0.06978628  0.01303309]\n",
              " [ 0.03871715  0.02608213  0.03544959 -0.02521311  0.11005433 -0.0143066\n",
              "  -0.03052466 -0.03852827  0.06321152  0.0038594 ]]\n",
              "<NDArray 2x10 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTmN2TyNIzuW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class FancyMLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(FancyMLP, self).__init__(**kwargs)\n",
        "        self.rand_weight = self.params.get_constant('rand_weight', nd.random.uniform(shape=(20, 20)))\n",
        "        self.dense = nn.Dense(20, activation='relu')\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.dense(x)\n",
        "        x = nd.relu(nd.dot(x, self.rand_weight.data()) + 1)\n",
        "        x = self.dense(x)\n",
        "        while x.norm().asscalar() > 1:\n",
        "            x /= 2 \n",
        "        if x.norm().asscalar() < 0.8:\n",
        "            x *= 10 \n",
        "        return x.sum()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AEDrl_jNtsyD",
        "colab_type": "code",
        "outputId": "05850e8f-6793-41b2-b3ee-b18a423aa033",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "net = FancyMLP()\n",
        "net.initialize()\n",
        "net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[18.571953]\n",
              "<NDArray 1 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2A9OvKS9txw1",
        "colab_type": "code",
        "outputId": "803deca5-7de9-4eb6-b139-2f63f53478f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "class NestMLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(NestMLP, self).__init__(**kwargs) \n",
        "        self.net = nn.Sequential()\n",
        "        self.net.add(nn.Dense(64, activation='relu'), \n",
        "              nn.Dense(32, activation='relu'))\n",
        "        self.dense = nn.Dense(16, activation='relu')\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.dense(self.net(x))\n",
        "\n",
        "net = nn.Sequential()\n",
        "net.add(NestMLP(), nn.Dense(20), FancyMLP())\n",
        "\n",
        "net.initialize()\n",
        "net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[24.86621]\n",
              "<NDArray 1 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rSHaFfBuu1Ix",
        "colab_type": "text"
      },
      "source": [
        "## 4.2 模型参数的访问、初始化和共享"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPbgjq-VvCZI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import init, nd \n",
        "from mxnet.gluon import nn\n",
        "\n",
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(256, activation='relu'))\n",
        "net.add(nn.Dense(10))\n",
        "net.initialize()\n",
        "\n",
        "X = nd.random.uniform(shape=(2, 20))\n",
        "Y = net(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZwE5FCtvfUQ",
        "colab_type": "code",
        "outputId": "1166338b-203c-4807-9c64-bd94e5119791",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dense(20 -> 256, Activation(relu))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RPlGakcIvrSZ",
        "colab_type": "code",
        "outputId": "16723328-ad6f-474b-dccd-01021b82065c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "net[0].params, type(net[0].params)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(dense11_ (\n",
              "   Parameter dense11_weight (shape=(256, 20), dtype=float32)\n",
              "   Parameter dense11_bias (shape=(256,), dtype=float32)\n",
              " ), mxnet.gluon.parameter.ParameterDict)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lxGfqmfdvyjA",
        "colab_type": "code",
        "outputId": "1ac5c1e5-8662-448a-bc8d-a8ea2b6723d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net[0].weight"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Parameter dense11_weight (shape=(256, 20), dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUIcWbV_ylLh",
        "colab_type": "code",
        "outputId": "8b9c42a6-5c89-489c-933d-3cbbe672c529",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "net[0].weight.data()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[-0.06046963  0.00624272 -0.03472826 ... -0.01759475  0.0686483\n",
              "  -0.06360765]\n",
              " [-0.01273243 -0.02659053 -0.04718638 ...  0.02570021  0.02275064\n",
              "  -0.0166979 ]\n",
              " [-0.03555115  0.01875034  0.02322027 ...  0.06564643  0.04601197\n",
              "  -0.01915742]\n",
              " ...\n",
              " [ 0.03173313  0.01789995  0.02519771 ... -0.06176154 -0.03986754\n",
              "  -0.04898471]\n",
              " [ 0.00564718  0.04665586 -0.00028374 ...  0.05332779  0.02100175\n",
              "  -0.06427249]\n",
              " [ 0.0438781   0.05357236  0.02753124 ...  0.04084889 -0.01963295\n",
              "   0.05668835]]\n",
              "<NDArray 256x20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aeLzHOQPysYg",
        "colab_type": "code",
        "outputId": "126842c4-8162-4a76-95bd-020177c448bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "net[0].weight.data()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[-0.06046963  0.00624272 -0.03472826 ... -0.01759475  0.0686483\n",
              "  -0.06360765]\n",
              " [-0.01273243 -0.02659053 -0.04718638 ...  0.02570021  0.02275064\n",
              "  -0.0166979 ]\n",
              " [-0.03555115  0.01875034  0.02322027 ...  0.06564643  0.04601197\n",
              "  -0.01915742]\n",
              " ...\n",
              " [ 0.03173313  0.01789995  0.02519771 ... -0.06176154 -0.03986754\n",
              "  -0.04898471]\n",
              " [ 0.00564718  0.04665586 -0.00028374 ...  0.05332779  0.02100175\n",
              "  -0.06427249]\n",
              " [ 0.0438781   0.05357236  0.02753124 ...  0.04084889 -0.01963295\n",
              "   0.05668835]]\n",
              "<NDArray 256x20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zk8Tlz03ywBC",
        "colab_type": "code",
        "outputId": "b62c8b6b-298a-455e-dbb7-f0898903e13f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        }
      },
      "source": [
        "net[0].weight.grad()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0. 0. 0. ... 0. 0. 0.]\n",
              " [0. 0. 0. ... 0. 0. 0.]\n",
              " [0. 0. 0. ... 0. 0. 0.]\n",
              " ...\n",
              " [0. 0. 0. ... 0. 0. 0.]\n",
              " [0. 0. 0. ... 0. 0. 0.]\n",
              " [0. 0. 0. ... 0. 0. 0.]]\n",
              "<NDArray 256x20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sHv-XvZzyzVX",
        "colab_type": "code",
        "outputId": "822ff8cd-66c6-4d09-8861-4ca5ff65de2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "net.collect_params()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "sequential2_ (\n",
              "  Parameter dense11_weight (shape=(256, 20), dtype=float32)\n",
              "  Parameter dense11_bias (shape=(256,), dtype=float32)\n",
              "  Parameter dense12_weight (shape=(10, 256), dtype=float32)\n",
              "  Parameter dense12_bias (shape=(10,), dtype=float32)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vErL03DIzh3g",
        "colab_type": "code",
        "outputId": "63ae1cac-f3c7-48f5-8a73-b8298b154cda",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "net.collect_params('.*weight')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "sequential2_ (\n",
              "  Parameter dense11_weight (shape=(256, 20), dtype=float32)\n",
              "  Parameter dense12_weight (shape=(10, 256), dtype=float32)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YmOUrxmLznsW",
        "colab_type": "code",
        "outputId": "df015efb-7777-4a4f-9ad6-513bf183692b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "net.initialize(init=init.Normal(sigma=0.01), force_reinit=True)\n",
        "net[0].weight.data()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[-0.00803235  0.01792648  0.00174623  0.01004736 -0.00177172  0.01704121\n",
              " -0.00315196 -0.0084464   0.00439474  0.00382651 -0.0071534  -0.01518173\n",
              " -0.00180067  0.01541854  0.00415876 -0.0093543   0.00476378 -0.00346679\n",
              "  0.00468796  0.01868755]\n",
              "<NDArray 20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AxZmkokqz53N",
        "colab_type": "code",
        "outputId": "8fb041e9-c452-4790-f677-4d0e0bab23f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        }
      },
      "source": [
        "net.initialize(init=init.Constant(1), force_reinit=True)\n",
        "net[0].weight.data()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[1. 1. 1. ... 1. 1. 1.]\n",
              " [1. 1. 1. ... 1. 1. 1.]\n",
              " [1. 1. 1. ... 1. 1. 1.]\n",
              " ...\n",
              " [1. 1. 1. ... 1. 1. 1.]\n",
              " [1. 1. 1. ... 1. 1. 1.]\n",
              " [1. 1. 1. ... 1. 1. 1.]]\n",
              "<NDArray 256x20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "evoUAGcO1IwL",
        "colab_type": "code",
        "outputId": "f179909c-b647-41ac-bd0d-601821ea25cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net[0].weight.data().shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(256, 20)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHf5K4W_1OGc",
        "colab_type": "code",
        "outputId": "01b47ba5-91e0-48fd-e9fb-792a4456282b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "net[0].weight.initialize(init=init.Xavier(), force_reinit=True)\n",
        "net[0].weight.data()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[-0.10389185  0.07822403 -0.1289716  -0.1410463  -0.07610903 -0.10696874\n",
              " -0.01996909 -0.07058676  0.00648634  0.10942626  0.08052795 -0.09453681\n",
              "  0.13527533 -0.01967503 -0.11284603 -0.05156991 -0.11588816  0.02459455\n",
              "  0.02644953  0.12870744]\n",
              "<NDArray 20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AqektfkA-o3",
        "colab_type": "code",
        "outputId": "4962772a-6a6f-4305-9995-c9d4d17d2f89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "class MyInit(init.Initializer):\n",
        "    def _init_weight(self, name, data):\n",
        "        print('Init', name, data.shape)\n",
        "        data[:] = nd.random.uniform(low=-10, high=10, shape=data.shape)\n",
        "        data *= data.abs() >= 5 \n",
        "\n",
        "net.initialize(MyInit(), force_reinit=True)\n",
        "net[0].weight.data()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Init dense11_weight (256, 20)\n",
            "Init dense12_weight (10, 256)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[ 7.142498  -9.206991  -7.6536884 -0.        -0.        -0.\n",
              " -0.        -5.294743  -0.        -8.822595   0.        -7.5231113\n",
              " -0.         7.135105   0.        -0.         0.        -0.\n",
              " -0.        -9.560527 ]\n",
              "<NDArray 20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D1dyk2_oBhZr",
        "colab_type": "code",
        "outputId": "68e5d47f-b793-4e8a-80b3-f85955509268",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "net[0].weight.set_data(net[0].weight.data() + 1)\n",
        "net[0].weight.data()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[ 8.142498  -8.206991  -6.6536884  1.         1.         1.\n",
              "  1.        -4.294743   1.        -7.8225946  1.        -6.5231113\n",
              "  1.         8.135105   1.         1.         1.         1.\n",
              "  1.        -8.560527 ]\n",
              "<NDArray 20 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNLLIApKCYFr",
        "colab_type": "code",
        "outputId": "b4322601-f4d9-470f-9f02-03677e83fd0a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "net = nn.Sequential()\n",
        "shared = nn.Dense(8, activation='relu')\n",
        "net.add(nn.Dense(8, activation='relu'), \n",
        "    shared, \n",
        "    nn.Dense(8, activation='relu', params=shared.params), \n",
        "    nn.Dense(10))\n",
        "net.initialize()\n",
        "\n",
        "X = nd.random.uniform(shape=(2, 20))\n",
        "net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
              "   0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
              "   0.0000000e+00  0.0000000e+00]\n",
              " [ 2.2168067e-06  2.6662779e-06 -3.6558160e-06 -1.7297840e-07\n",
              "  -2.8038012e-06  3.0073204e-06  2.0541652e-06  1.5204251e-06\n",
              "  -4.2007518e-06  3.4896932e-06]]\n",
              "<NDArray 2x10 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8x6DPWZeC9I8",
        "colab_type": "code",
        "outputId": "9b58fb3d-7634-4d3c-cbc9-11bc517fa840",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "net[1].weight.data()[0] == net[2].weight.data()[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[1. 1. 1. 1. 1. 1. 1. 1.]\n",
              "<NDArray 8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s7jOjy-NDHhI",
        "colab_type": "text"
      },
      "source": [
        "## 4.3 模型参数的延后初始化"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80Sj5dJFDQnA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import init, nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "class MyInit(init.Initializer):\n",
        "    def _init_weight(self, name, data):\n",
        "        print('Init', name, data.shape)\n",
        "\n",
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(256, activation='relu'), \n",
        "    nn.Dense(10))\n",
        "\n",
        "net.initialize(init=MyInit())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0HkPWsV_EdZ3",
        "colab_type": "code",
        "outputId": "75dc4e3a-5221-4549-a29d-71a24431f190",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "X = nd.random.uniform(shape=(2, 20))\n",
        "Y = net(X)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Init dense19_weight (256, 20)\n",
            "Init dense20_weight (10, 256)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHgQlzR2QxTo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Y = net(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0JdrQDGxQ_ez",
        "colab_type": "code",
        "outputId": "9f8c974e-9e5c-4eca-9716-68a094caaf25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "net.initialize(init=MyInit(), force_reinit=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Init dense19_weight (256, 20)\n",
            "Init dense20_weight (10, 256)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q_ELNBfMRJX2",
        "colab_type": "code",
        "outputId": "20d1d913-0596-49df-84c3-b8883c6cbf5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(256, in_units=20, activation='relu'))\n",
        "net.add(nn.Dense(10, in_units=256))\n",
        "\n",
        "net.initialize(init=MyInit())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Init dense21_weight (256, 20)\n",
            "Init dense22_weight (10, 256)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jdCzuaf6RZ8G",
        "colab_type": "text"
      },
      "source": [
        "## 自定义层"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tpvSTxhtRf0e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import gluon, nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "class CenteredLayer(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(CenteredLayer, self).__init__(**kwargs)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        return x - x.mean()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XBTbridCR7Qd",
        "colab_type": "code",
        "outputId": "d916fdcd-8cd2-45f9-8060-988adf3aea80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "layer = CenteredLayer()\n",
        "layer(nd.array([1, 2, 3, 4, 5]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[-2. -1.  0.  1.  2.]\n",
              "<NDArray 5 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5lMVJISaSBD0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(128), \n",
        "    CenteredLayer())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "61BzMwVISJ-9",
        "colab_type": "code",
        "outputId": "597c08d2-17b6-467c-8c08-e5f0380f2746",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "net.initialize()\n",
        "y = net(nd.random.uniform(shape=(4, 8)))\n",
        "y.mean().asscalar()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-3.6379788e-11"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZSgz-4jSTW3",
        "colab_type": "code",
        "outputId": "f7c03ba4-f226-43d0-b06d-a67d4bd4c9de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "params = gluon.ParameterDict()\n",
        "params.get('param2', shape=(2, 3))\n",
        "params "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\n",
              "  Parameter param2 (shape=(2, 3), dtype=<class 'numpy.float32'>)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0YsYjWlRSm9M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyDense(nn.Block):\n",
        "    def __init__(self, units, in_units, **kwargs):\n",
        "        super(MyDense, self).__init__(**kwargs)\n",
        "        self.weight = self.params.get('weight', shape=(in_units, units))\n",
        "        self.bias = self.params.get('bias', shape=(units, ))\n",
        "\n",
        "    def forward(self, x):\n",
        "        linear = nd.dot(x, self.weight.data()) + self.bias.data()\n",
        "        return nd.relu(linear)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uaWx4ONxVqMO",
        "colab_type": "code",
        "outputId": "d7d85458-7b7a-48e9-b7de-8b422df189ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "dense = MyDense(units=3, in_units=5)\n",
        "dense.params"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "mydense0_ (\n",
              "  Parameter mydense0_weight (shape=(5, 3), dtype=<class 'numpy.float32'>)\n",
              "  Parameter mydense0_bias (shape=(3,), dtype=<class 'numpy.float32'>)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAnojIunVwe_",
        "colab_type": "code",
        "outputId": "637ee6d5-521f-4f64-b1cd-708722c6e90e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "dense.initialize()\n",
        "dense(nd.random.uniform(shape=(2, 5)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.04578072 0.05299256 0.02392543]\n",
              " [0.05540341 0.05370332 0.01740249]]\n",
              "<NDArray 2x3 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "spvhOmaGbOIG",
        "colab_type": "code",
        "outputId": "6654a3bd-cc79-472e-f283-83171e1f7451",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(MyDense(8, in_units=64), \n",
        "    MyDense(1, in_units=8))\n",
        "net.initialize()\n",
        "net(nd.random.uniform(shape=(2, 64)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.04486297]\n",
              " [0.05108206]]\n",
              "<NDArray 2x1 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3iAtO-A2bgMy",
        "colab_type": "text"
      },
      "source": [
        "## 4.5 读取和存储"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pum7qbtgblMV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "x = nd.ones(3)\n",
        "nd.save('x', x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_NQ8aR4bvcQ",
        "colab_type": "code",
        "outputId": "b647e7e3-d55d-4767-d6e3-2eabf882074a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "x2 = nd.load('x')\n",
        "x2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[\n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exjRsAqVbyws",
        "colab_type": "code",
        "outputId": "3c5bfca5-54a5-447a-99f1-2f161e6cb504",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "source": [
        "y = nd.zeros(4)\n",
        "nd.save('xy', [x, y])\n",
        "x2, y2 = nd.load('xy')\n",
        "(x2, y2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(\n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>, \n",
              " [0. 0. 0. 0.]\n",
              " <NDArray 4 @cpu(0)>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9gTRll0eSpe",
        "colab_type": "code",
        "outputId": "85f7736a-8b1f-4685-a14c-82d6276b15fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        }
      },
      "source": [
        "mydict = {'x': x, 'y': y}\n",
        "nd.save('mydict', mydict)\n",
        "mydict2 = nd.load('mydict')\n",
        "mydict2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'x': \n",
              " [1. 1. 1.]\n",
              " <NDArray 3 @cpu(0)>, 'y': \n",
              " [0. 0. 0. 0.]\n",
              " <NDArray 4 @cpu(0)>}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8onvtFWmeq5H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MLP(nn.Block):\n",
        "    def __init__(self, **kwargs):\n",
        "        super(MLP, self).__init__(**kwargs)\n",
        "        self.hidden = nn.Dense(256, activation='relu')\n",
        "        self.output = nn.Dense(10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.output(self.hidden(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XGQBDMBRfRCw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = MLP()\n",
        "net.initialize()\n",
        "X = nd.random.uniform(shape=(2, 20))\n",
        "Y = net(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8vcvKQDfaYF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "filename = 'mlp.params'\n",
        "net.save_parameters(filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMCDvpyhfhEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net2 = MLP()\n",
        "net2.load_parameters(filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrnjedYrfn5O",
        "colab_type": "code",
        "outputId": "4217a36b-eef4-43a4-fc18-fc92a691ad72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "Y2 = net2(X)\n",
        "Y2 == Y"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]\n",
              " [1. 1. 1. 1. 1. 1. 1. 1. 1. 1.]]\n",
              "<NDArray 2x10 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OqHzf2xQfrjC",
        "colab_type": "text"
      },
      "source": [
        "## 4.6 GPU 计算"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLQZSIgu-GnJ",
        "colab_type": "code",
        "outputId": "5c02dfae-19ff-4a35-e55c-0c4e6d0cb280",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Dec  3 19:10:24 2019       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.33.01    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmOBkRL4-L7E",
        "colab_type": "code",
        "outputId": "9d3f2133-53c3-4859-f8f5-d76a5b93e41c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!cat /usr/local/cuda/version.txt"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CUDA Version 10.0.130\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OwzwbiSHtnZ",
        "colab_type": "code",
        "outputId": "957b69cc-5765-4006-a7c7-8d894cc34a0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171
        }
      },
      "source": [
        "!pip install mxnet-cu100"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: mxnet-cu100 in /usr/local/lib/python3.6/dist-packages (1.5.1.post0)\n",
            "Requirement already satisfied: numpy<2.0.0,>1.16.0 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (1.17.4)\n",
            "Requirement already satisfied: requests<3,>=2.20.0 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (2.21.0)\n",
            "Requirement already satisfied: graphviz<0.9.0,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from mxnet-cu100) (0.8.4)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet-cu100) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet-cu100) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet-cu100) (2019.9.11)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.20.0->mxnet-cu100) (2.8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAa9UQo7HXyv",
        "colab_type": "code",
        "outputId": "77c8ec01-7f8b-4800-c9de-dce649368ebc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import mxnet as mx \n",
        "from mxnet import nd \n",
        "from mxnet.gluon import nn \n",
        "\n",
        "mx.cpu(), mx.gpu(), mx.gpu(1)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(cpu(0), gpu(0), gpu(1))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdFNgiMdHp_z",
        "colab_type": "code",
        "outputId": "0d16eee2-6e2e-4599-d747-c81dfc2e2eb9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "x = nd.array([1, 2, 3])\n",
        "x"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[1. 2. 3.]\n",
              "<NDArray 3 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nCq0ben3IJmi",
        "colab_type": "code",
        "outputId": "a84f2ea6-ecea-4cc8-c46f-124c57ab3179",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x.context"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "cpu(0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gX5SMZL0IMcB",
        "colab_type": "code",
        "outputId": "a9003ce1-a0cc-42a8-cbf6-506b8cfc8776",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "a = nd.array([1, 2, 3], ctx=mx.gpu())\n",
        "a"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[1. 2. 3.]\n",
              "<NDArray 3 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RV0GuWn3ISPZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 528
        },
        "outputId": "a72a0fd4-e08f-4681-e34a-aeeaa3c914aa"
      },
      "source": [
        "B = nd.random.uniform(shape=(2, 3), ctx=mx.gpu(1))\n",
        "B"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "error",
          "ename": "MXNetError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mMXNetError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-9-2df5512b45c0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/mxnet/ndarray/random.py\u001b[0m in \u001b[0;36muniform\u001b[0;34m(low, high, shape, dtype, ctx, out, **kwargs)\u001b[0m\n\u001b[1;32m    108\u001b[0m     \"\"\"\n\u001b[1;32m    109\u001b[0m     return _random_helper(_internal._random_uniform, _internal._sample_uniform,\n\u001b[0;32m--> 110\u001b[0;31m                           [low, high], shape, dtype, ctx, out, kwargs)\n\u001b[0m\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/mxnet/ndarray/random.py\u001b[0m in \u001b[0;36m_random_helper\u001b[0;34m(random, sampler, params, shape, dtype, ctx, out, kwargs)\u001b[0m\n\u001b[1;32m     46\u001b[0m                 \u001b[0;34m\"Distribution parameters must all have the same type, but got \"\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m                 \u001b[0;34m\"both %s and %s.\"\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m     raise ValueError(\"Distribution parameters must be either NDArray or numbers, \"\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/mxnet/ndarray/register.py\u001b[0m in \u001b[0;36m_random_uniform\u001b[0;34m(low, high, shape, ctx, dtype, out, name, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/mxnet/_ctypes/ndarray.py\u001b[0m in \u001b[0;36m_imperative_invoke\u001b[0;34m(handle, ndargs, keys, vals, out)\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0mc_str_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0mc_str_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m         ctypes.byref(out_stypes)))\n\u001b[0m\u001b[1;32m     93\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0moriginal_output\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/mxnet/base.py\u001b[0m in \u001b[0;36mcheck_call\u001b[0;34m(ret)\u001b[0m\n\u001b[1;32m    251\u001b[0m     \"\"\"\n\u001b[1;32m    252\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 253\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mMXNetError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_LIB\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMXGetLastError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mMXNetError\u001b[0m: [19:11:13] src/engine/threaded_engine.cc:330: Check failed: exec_ctx.dev_id < device_count_ (1 vs. 1) : Invalid GPU Id: 1, Valid device id should be less than device_count: 1\nStack trace:\n  [bt] (0) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x4b04cb) [0x7f76606d74cb]\n  [bt] (1) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x25b4f41) [0x7f76627dbf41]\n  [bt] (2) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x25be491) [0x7f76627e5491]\n  [bt] (3) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x2e72505) [0x7f7663099505]\n  [bt] (4) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x2e726f7) [0x7f76630996f7]\n  [bt] (5) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x2e7b7c3) [0x7f76630a27c3]\n  [bt] (6) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(+0x2e7d09e) [0x7f76630a409e]\n  [bt] (7) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(mxnet::imperative::SetDependency(nnvm::NodeAttrs const&, mxnet::Context const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> >*, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> >*, std::vector<mxnet::Resource, std::allocator<mxnet::Resource> >*, std::vector<unsigned int, std::allocator<unsigned int> >*, mxnet::DispatchMode)+0xf76) [0x7f76628923a6]\n  [bt] (8) /usr/local/lib/python3.6/dist-packages/mxnet/libmxnet.so(mxnet::Imperative::InvokeOp(mxnet::Context const&, nnvm::NodeAttrs const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, mxnet::DispatchMode, mxnet::OpStatePtr)+0x18a) [0x7f7662892b6a]\n\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R0k1VemOLDyI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "72461523-5ab5-4f43-f255-56ac822d5f47"
      },
      "source": [
        "y = x.copyto(mx.gpu())\n",
        "y"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[1. 2. 3.]\n",
              "<NDArray 3 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oxFPkwomLQ1v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "a70b7233-62a5-4d34-e834-3e472a5554c0"
      },
      "source": [
        "z = x.as_in_context(mx.gpu())\n",
        "z"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[1. 2. 3.]\n",
              "<NDArray 3 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VU7YjBlILYZi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "904e60b9-240b-4471-cf35-9bff1641c137"
      },
      "source": [
        "y.as_in_context(mx.gpu()) is y "
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zrfj9zFrLj96",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "133fe9cd-c862-4e2c-deac-80a17bb47195"
      },
      "source": [
        "y.copyto(mx.gpu()) is y "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UqizBB8yLn_q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "b337e28f-3650-4a55-dd78-0e07297f15b3"
      },
      "source": [
        "(z + 2).exp() * y"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[ 20.085537 109.1963   445.2395  ]\n",
              "<NDArray 3 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_thkdS31LsZ6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(1))\n",
        "net.initialize(ctx=mx.gpu())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WpA4fTosMDiC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "acf29a2c-72a9-4d91-bbef-d852ecf89707"
      },
      "source": [
        "net(y)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.0068339 ]\n",
              " [0.01366779]\n",
              " [0.02050169]]\n",
              "<NDArray 3x1 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vbxUdcZHMEk5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "9d6552a5-446c-4d0c-cc7d-aca85c436349"
      },
      "source": [
        "net[0].weight.data()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.0068339]]\n",
              "<NDArray 1x1 @gpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    }
  ]
}